﻿
//Copyright 1997-2009 Syrinx Development, Inc.
//This file is part of the Syrinx Web Application Framework (SWAF).
// == BEGIN LICENSE ==
//
// Licensed under the terms of any of the following licenses at your
// choice:
//
//  - GNU General Public License Version 3 or later (the "GPL")
//    http://www.gnu.org/licenses/gpl.html
//
//  - GNU Lesser General Public License Version 3 or later (the "LGPL")
//    http://www.gnu.org/licenses/lgpl.html
//
//  - Mozilla Public License Version 1.1 or later (the "MPL")
//    http://www.mozilla.org/MPL/MPL-1.1.html
//
// == END LICENSE ==
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;

using Swaf;
using Swaf.BizObj;
using Swaf.BizRule;
using Swaf.Call;
using Swaf.Container;
using Swaf.Scheduler;
using Swaf.Security;

namespace Forums
{
	public class ForumScarfBizRules : Syrinx.BaseDataScarfBizRules
	{
		protected void setupContext(IRuleContext context, bool allowTranslations)
		{
			context.put("doingArticleRestore", true);
            context.put("delayTranslations", !allowTranslations);
            //context.put("delayTranslations", true);
        }
		public void scarfForumSet(IApplication app, IRuleContext context, object sinfo)
		{
			IBizObj scarfInfo = null;
			if (sinfo is IBizObj)
				scarfInfo = (IBizObj)sinfo;
			else if (sinfo is string)
				scarfInfo = app.appData.get("ForumScarfs." + sinfo) as IBizObj;

			if (scarfInfo == null)
				return;
            scarfInfo = (IBizObj)scarfInfo.Clone();

            string lfpFile = null;
            IBizObjList lastForumPosts = null;
            if (!Cmn.IsEmpty(scarfInfo.get("saveRuntimeLocation")))
            {
                FileInfo f = new FileInfo(app.resMgr.resolve((string)scarfInfo.get("saveRuntimeLocation")));
                DirectoryInfo d = f.Directory;
                lfpFile = Cmn.pathCombine(d.FullName, "LastForumPosts.xml");
                if (File.Exists(lfpFile))
                {
                    string xml = File.ReadAllText(lfpFile);
                    lastForumPosts = app.bizObjMgr.createList(SourceType.BasicSource, xml, "LastForumPostInfo", null);
                }
                else
                    lastForumPosts = app.bizObjMgr.createList("LastForumPostInfo");
                context.put("lfp", lastForumPosts.Clone());
            }

			DateTime startTime = DateTime.Now;
			setupContext(context, true);
			scarfInfo.put("runtime.startTime", startTime);
			scarfInfo.put("lastSaveTime", startTime);
			writeTrace(app, scarfInfo, "<ForumScarf id='{0}' event='started' startTime='{1}' />", scarfInfo.get("id"), startTime);

			//Step 0: Check for login info and use it
			if (!Cmn.IsEmpty(scarfInfo.get("login.url")))
			{
				writeTrace(app, scarfInfo, "<ForumScarf id='{0}' event='initiateLogin' userId='{1}' />", scarfInfo.get("id"), scarfInfo.get("siteUserId"));
				loginUser(scarfInfo);
			}

			//Step 1: Get the home page into a string
            string forumSetPage = getPageText((string)scarfInfo.get("forumGroupsUrl"), scarfInfo);

			IBizObjList forumList = app.bizObjMgr.createList("Article");
			string forumRootUniqueId = scarfInfo.get("id") + "0_";
			Swaf.BGuid.Guid forumSetArticleId = context.RuleGroup.execute(context, "getArticleIdFromUniqueTitle", forumRootUniqueId, "Forum").CallResults as Swaf.BGuid.Guid;
			bool isNewForumSet = forumSetArticleId == null;
			if (isNewForumSet)
			{
				writeTrace(app, scarfInfo, "<ForumScarf id='{0}' event='createNewForumSet' forumRootUniqueId='{1}' />", scarfInfo.get("id"), forumRootUniqueId);
				IBizObj setBizObj = setupForumBizObj(context, app.bizObjMgr.create("Article"), scarfInfo, "", null, 0, (string)scarfInfo.get("id"), null, null, null, false, null, true);
				forumSetArticleId = (Swaf.BGuid.Guid)setBizObj.get("id");
			}
			else
				writeTrace(app, scarfInfo, "<ForumScarf id='{0}' event='addToExistingForumSet' forumRootUniqueId='{1}' />", scarfInfo.get("id"), forumRootUniqueId);

			bool fullForumScan = (bool)scarfInfo.get("fullForumScan");


			//Step 2: Strip groups and forums from page
            string ignoreForumGroups = (string)scarfInfo.get("ignoreForumGroups");
            string ignoreForums = (string)scarfInfo.get("ignoreForums");

            IRuleGroup forumScarf = null;
			Regex groupRe = new Regex((string)scarfInfo.get("groupRegex"), RegexOptions.Singleline);
			Regex forumRe = new Regex((string)scarfInfo.get("forumRegex"), RegexOptions.Singleline);
			MatchCollection mc = groupRe.Matches(forumSetPage);
			writeTrace(app, scarfInfo, "<ForumScarf id='{0}' event='forumGroupsFound' count='{1}' />", scarfInfo.get("id"), mc.Count);
			for (int pos = 0; pos < mc.Count; pos++)
			{
				Match m = mc[pos];
				string groupId = m.Groups[1].Value;
				string groupName = pos.ToString("D2") + "~!" + m.Groups[2].Value;
				int startPos = m.Index + m.Length;
				int endPos = (pos < mc.Count - 1) ? endPos = mc[pos + 1].Index : -1;
                if (ignoreForumGroups.Contains("," + groupId + ","))
                    continue;

				IBizObj groupBizObj = setupForumBizObj(context, forumList.insertNew(), scarfInfo,
					groupId, forumSetArticleId,1, groupName, "", null, null,true,null, true);


				MatchCollection mc2 = forumRe.Matches(endPos == -1 ? forumSetPage.Substring(startPos) : forumSetPage.Substring(startPos, endPos - startPos));
				for(int pos2 = 0; pos2 < mc2.Count; pos2++)
				{
					Match m2 = mc2[pos2];
					string forumId = m2.Groups["id"].Value;
					string forumName = pos2.ToString("D2") + "~!" + m2.Groups["title"].Value;
					string forumDesc = m2.Groups["desc"].Value;
					string subForums = m2.Groups["subForums"].Value;
					string lastPostDateStr = fixupDateStr(m2.Groups["lastPostDate"].Value);
					DateTime lastPostDate = DateTime.MinValue;
					DateTime.TryParse(lastPostDateStr, out lastPostDate);
                    if (ignoreForums.Contains("," + forumId + ","))
                        continue;

                    if (lastForumPosts != null)
                    {
                        IBizObj lfp = lastForumPosts.search("forumId", forumId, false);
                        if (lfp != null && lastPostDateStr.CompareTo(lfp.get("lastPostTime")) == 0)
                            continue;
                        if (lfp == null)
                        {
                            lfp = lastForumPosts.insertNew();
                            lfp.put("forumId", forumId);
                        }
                        lfp.put("lastPostTime", lastPostDateStr);
                    }

					IBizObj forumBizObj = setupForumBizObj(context, forumList.insertNew(), scarfInfo,
						forumId, groupBizObj.get("id"), 2, forumName, forumDesc, null, null,true,null,true);

                    if (Cmn.IsFalse(scarfInfo.get("useParallel")))
						app.ruleMgr.execute(context, "fullyScarfForumThreads", scarfInfo, forumBizObj, forumId);
					else if (forumScarf == null)
						forumScarf = app.ruleMgr.executeAsync(context, "fullyScarfForumThreads", null, null, new object[] { scarfInfo, forumBizObj, forumId });
					else
						forumScarf.executeAsync(context, "fullyScarfForumThreads", null, null, new object[] { scarfInfo, forumBizObj, forumId });
				}
			}
			if (forumScarf != null)
				forumScarf.waitForCompletion(true);

			DateTime endTime = DateTime.Now;
			scarfInfo.put("runtime.endTime", endTime);


            if (!Cmn.IsEmpty(lfpFile) && lastForumPosts != null)
                File.WriteAllText(lfpFile, lastForumPosts.Xml);

            string saveRuntime = (string)scarfInfo.get("saveRuntimeLocation");
            System.IO.File.WriteAllText(app.resMgr.resolve(saveRuntime), ((IBizObj)scarfInfo.get("runtime")).Xml);

            if (!Cmn.IsEmpty(scarfInfo.get("schedulerRange")))
            {
                string[] range = scarfInfo.get("schedulerRange").ToString().Split(',');
                int lowNum = int.Parse(range[0]);
                int highNum = lowNum;
                if(range.Length > 1)
                    highNum = int.Parse(range[1]);

                int actualNum = lowNum;
                if (lowNum != highNum)
                    actualNum = new Random().Next(lowNum, highNum);

                IScheduleManager scheduleManager = (IScheduleManager)app.appData["sysObjs-Scheduler"];
                string freq = string.Format("in {0} min", actualNum);
                string taskId = "scarfForum" + scarfInfo.get("id");
                ICall ruleCall = new RuleMgrCall(taskId, "scarfForumSet", sinfo);
                scheduleManager.createScheduledTask(taskId, freq, ruleCall);               
            }
		}


		protected IBizObj setupForumBizObj(IRuleContext context, IBizObj obj, IBizObj scarfInfo, string extId, object parentId, 
			int depth, string title, string summary, string body, string updatedDate, bool forceUpdate, Swaf.BGuid.Guid userId,
            bool allowTranslations)
		{
			setupContext(context, allowTranslations);
			IApplication app = Application.currentApp;
			if (obj.isNew)
			{
				string uniqueTitle = scarfInfo.get("id") + depth.ToString() + "_" + extId;
				Swaf.BGuid.Guid objId = app.ruleMgr.execute("getArticleIdFromUniqueTitle", uniqueTitle, "Forum").CallResults as Swaf.BGuid.Guid;
				bool isKnown = objId != null;
                if (!isKnown)
                {
                    objId = app.ruleMgr.execute("getArticleIdFromUniqueTitle", uniqueTitle, "DeletedForum").CallResults as Swaf.BGuid.Guid;
                    isKnown = objId != null;
                    if (!isKnown)
                        objId = new Swaf.BGuid.Guid();
                }

                if (isKnown)
                    obj.isNew = false;

				obj.put("id", objId);
				obj.put("uniqueTitle", uniqueTitle);
                if(Cmn.IsEmpty(obj.get("treeName")))
				    obj.put("treeName", "Forum");

				if((parentId == null && obj.get("parentId") == null) ||
					(parentId != null && ((IComparable)parentId).CompareTo(obj.get("parentId"))!=0))
					obj.put("parentId", parentId);
				if (isKnown)
				{
					FlexiMap m = new FlexiMap();
					m.put("node", obj);
					IBizObj info = app.appData.get("DA.Syrinx.GetNodeInfoInsert", m) as IBizObj;
					if (info == null)
						m.put("a", "For debugging");
				}
				else
				{
					DateTime min = new DateTime(1753, 1, 1);
					obj.put("counts.dateUpdated", min);
					obj.put("counts.dateCreated", min);
					obj.put("counts.lastChildUpdated", min);
				}
				obj.put("depth", depth);
			}
			if (title != null && title.CompareTo(obj.get("baseTitle")) != 0)
				obj.put("baseTitle", title);
			if(summary != null)
				obj.put("baseSummary", summary);
			if (body != null)
				obj.put("baseBody", body);

			if (userId != null)
				obj.put("counts.author.id", userId);
			if (!Cmn.IsEmpty(updatedDate))
			{
				string date = updatedDate.Trim();
				DateTime dt;
				if (DateTime.TryParse(date, out dt))
				{
                    int hrsOffOfUTC = (int)scarfInfo.get("timeDiffFromUTC");
                    dt = dt.AddHours(hrsOffOfUTC * -1);
					obj.put("counts.dateUpdated", dt);
					obj.put("counts.dateCreated", dt);
				}
			}

			if (obj.isNew || forceUpdate)
			{
				IResults rc = context.RuleGroup.execute(context, "updateArticle", obj);
			}
			return obj;
		}


		public void fullyScarfForumThreads(IApplication app, IRuleContext context, IBizObj scarfInfo, IBizObj forumBizObj, string extId)
		{
			//Step 1: Get first page of forum threads
			string url = (string)scarfInfo.get("forumPageUrl");
			url = url.Replace("$extId$", extId);
			string forumPage = getPageText(url, scarfInfo);

			//Step 2: get total number of pages.
			Regex pagesRe = new Regex((string)scarfInfo.get("forumPagesRegex"), RegexOptions.Singleline);
			Match m = pagesRe.Match(forumPage);
			int totalPages = 1;
			if (m.Success)
				totalPages = int.Parse(m.Groups["lastPage"].Value);
			writeTrace(app, scarfInfo, "<ForumScarf id='{0}' event='scanAllForumThreads' numPages='{1}' forumTitle='{2}' />", scarfInfo.get("id"), totalPages, forumBizObj.get("title"));

			//step 3: parallel call a bizrule to process each page's threads.
			bool getNextPage = scarfThreads(app,context, scarfInfo, forumBizObj.get("id"), forumPage, true, 1, totalPages);

			bool fullForumScan = (bool)scarfInfo.get("fullForumScan"); 
			if (totalPages > 1 && (fullForumScan || getNextPage))
				if (Cmn.IsFalse(scarfInfo.get("useParallel")))
					context.RuleGroup.execute(context, "scarfForumThreadPage", scarfInfo, 2, totalPages, forumBizObj, extId);
				else
					context.RuleGroup.executeAsync(context, "scarfForumThreadPage", null, null, new object[] { scarfInfo, 2, totalPages, forumBizObj, extId });

			//Step 4: update last scarf date
		}

		protected bool scarfThreads(IApplication app, IRuleContext context, IBizObj scarfInfo, object forumId, string html, bool pullPosts, int page, int totalPages)
		{
            string ignoreUserIds = (string)scarfInfo.get("ignoreUsers");
            string lastForumPostTime = null;
            DateTime lfpTime = DateTime.MinValue;
            IBizObjList lastForumPosts = context.get("lfp") as IBizObjList;
            if (lastForumPosts != null)
            {
                IBizObj ft = lastForumPosts.search("forumId", forumId, false);
                if (ft != null)
                {
                    lastForumPostTime = (string)ft.get("lastPostTime");
                    if(DateTime.TryParse(lastForumPostTime, out lfpTime))
                        lfpTime=lfpTime.AddHours((int)scarfInfo.get("timeDiffFromUTC"));
                }
            }
			IBizObjList threads = app.bizObjMgr.createList("Article");
			Regex threadRe = new Regex((string)scarfInfo.get("threadRegex"), RegexOptions.Singleline);
			MatchCollection mc = threadRe.Matches(html);
			bool doNextPage = true;

			if((int)scarfInfo.get("expectedThreadsPerPage") != mc.Count && page != totalPages)
				writeTrace(app, scarfInfo, "<ForumScarf id='{0}' event='forumThreadsFound-Issue' numThreads='{1}' expectedThreads='{2}' forumId='{3}' threadPage='{4}' totalPages='{5}' pageSize='{6}' />",
					scarfInfo.get("id"), mc.Count, scarfInfo.get("expectedThreadsPerPage"), forumId, page, totalPages, html.Length);
			else
				writeTrace(app, scarfInfo, "<ForumScarf id='{0}' event='forumThreadsFound' numThreads='{1}' forumId='{2}' threadPage='{3}' pageSize='{4}' />", scarfInfo.get("id"), mc.Count,forumId, page, html.Length);
			lock (scarfInfo)
			{
				scarfInfo.put("runtime.totalThreadsFound", ((int)scarfInfo.get("runtime.totalThreadsFound")) + mc.Count);
			}

			string threadId = "Unknown";
			string threadTitle = "Unknown";
			try
			{
				foreach (Match m in mc)
				{
                    string threadUserId = m.Groups["threadUserId"].Value;
                    if (ignoreUserIds.Contains("," + threadUserId + ","))
                        continue;
                    threadId = m.Groups["id"].Value;
					threadTitle = m.Groups["title"].Value.Trim();
					string lastPostDateStr = fixupDateStr(m.Groups["lastUpdated"].Value.Trim());
					int replies = int.Parse(m.Groups["replies"].Value);
					DateTime lastPostDate = DateTime.MinValue;
					DateTime.TryParse(lastPostDateStr, out lastPostDate);
					IBizObj threadBizObj = setupForumBizObj(context, threads.insertNew(), scarfInfo, threadId, forumId, 3, threadTitle, null, null, null, false, null, false);
					DateTime lastForumPost = (DateTime)threadBizObj.get("counts.lastChildUpdated");
                    if (lastForumPostTime != null)
                        lastForumPost = lfpTime;
                    else
                        lastForumPost = lastForumPost.AddHours((int)scarfInfo.get("timeDiffFromUTC"));

                    if (pullPosts && lastPostDate > lastForumPost)
                    {
                        doNextPage = true;
                        threadBizObj.isNew = false;
                        threadBizObj.clearDirtyFlags();
                        if (Cmn.IsFalse(scarfInfo.get("useParallel")))
                            app.ruleMgr.execute(context, "scarfForumThreadPosts", scarfInfo, threadBizObj, replies + 1);
                        else
                            app.ruleMgr.executeAsync(context, "scarfForumThreadPosts", null, null, new object[] { scarfInfo, threadBizObj, replies + 1 });
                    }
                    else
                    {
                        doNextPage = false;
                        //break;
                    }
				}
			}
			catch (Exception e)
			{
				writeTrace(app, scarfInfo, "<ForumScarf id='{0}' event='scarfThreadFailure' threadId='{1}' threadTitle='{2}' />", scarfInfo.get("id"), threadId, threadTitle);
			}

			return doNextPage;
		}

		public void scarfForumThreadPage(IApplication app, IRuleContext context, IBizObj scarfInfo, int pageIndex, int totalPages, IBizObj forumBizObj, string extId)
		{
			int multiplier = (int)scarfInfo.get("forumPageIndexMultiplier");
			int i = pageIndex;
			if (multiplier > 1)
				i = (multiplier * (pageIndex - 1));

			//Step 1: Get forum thread page into string
			string url = (string)scarfInfo.get("forumPageUrl");
			url = url.Replace("$extId$", extId);
			string pagedUrl = url + ((string)scarfInfo.get("forumPagePageIndexInfo")).Replace("$pageIndex$", i.ToString());
			string forumPage = getPageText(pagedUrl, scarfInfo);

			bool getNextPage = scarfThreads(app, context, scarfInfo, forumBizObj.get("id"), forumPage, true, pageIndex, totalPages);
			bool fullForumScan = (bool)scarfInfo.get("fullForumScan");
			if (pageIndex < totalPages && (fullForumScan || getNextPage))
			{
				if (Cmn.IsFalse(scarfInfo.get("useParallel")))
				{
					writeTrace(app, scarfInfo, "<ForumScarf id='{0}' event='scarfForumThreadPage-NextPageSequential' forumId='{1}' pageIndex='{2}' totalPages='{3}' />", scarfInfo.get("id"), extId, pageIndex, totalPages);
					context.RuleGroup.execute(context, "scarfForumThreadPage", scarfInfo, ++pageIndex, totalPages, forumBizObj, extId);
				}
				else
				{
					writeTrace(app, scarfInfo, "<ForumScarf id='{0}' event='scarfForumThreadPage-NextPageParallel' forumId='{1}' pageIndex='{2}' totalPages='{3}' />", scarfInfo.get("id"), extId, pageIndex, totalPages);
					context.RuleGroup.executeAsync(context, "scarfForumThreadPage", null, null, new object[] { scarfInfo, ++pageIndex, totalPages, forumBizObj, extId });
				}
			}
			else
			{
				writeTrace(app, scarfInfo, "<ForumScarf id='{0}' event='scarfForumThreadPage-Done' forumId='{1}' pageIndex='{2}' totalPages='{3}' />", scarfInfo.get("id"), extId, pageIndex, totalPages );
			}
		}

		public void scarfForumThreadPosts(IApplication app, IRuleContext context, IBizObj scarfInfo, IBizObj threadBizObj, int expectedPosts)
		{
			string threadKeyName = (string)threadBizObj.get("keyName");
			//Step 1: get first page of thread posts.
			string extId = (string)threadBizObj.get("uniqueTitle");
			extId = extId.Substring(extId.LastIndexOf('_')+1);
			string url = (string)scarfInfo.get("threadPageUrl");
			url = url.Replace("$extId$", extId);
			string threadPage = getPageText(url, scarfInfo);
            string ignoreUserIds = (string)scarfInfo.get("ignoreUsers");

			//Step 2: Calculate # of pages.
			Regex pagesRe = new Regex((string)scarfInfo.get("threadPagesRegex"), RegexOptions.Singleline);
			Match m = pagesRe.Match(threadPage);
			int totalPages = 1;
			if (m.Success)
				if (!int.TryParse(m.Groups["lastPage"].Value, out totalPages))
					totalPages = 1;

			//Step 3: If newThread, foreach post in thread, add moving forward through pages

			//Step 4: If not newthread, go to last page and look for post that is in db because
			//       its post date is < last scarf date.  Keep going back in pages until found
			//       then move forward through posts add the new ones.  this allows code to add
			//       posts into thread in the order they were created in forum.
			IBizObjList posts = app.bizObjMgr.createList("Article");
			int multiplier = (int)scarfInfo.get("threadPageIndexMultiplier");
			int totalPostsFound = 0;
			for (int pagePos = 1; pagePos <= totalPages; pagePos++)
			{
				string postPage = threadPage;
				if (pagePos > 1)
				{
					int i = pagePos;
					if (multiplier > 1)
						i = (multiplier * (pagePos - 1));
					string pagedUrl = url + ((string)scarfInfo.get("threadPagePageIndexInfo")).Replace("$pageIndex$", i.ToString());
					postPage = getPageText(pagedUrl, scarfInfo);
				}

				Regex threadRe = new Regex((string)scarfInfo.get("postRegex"), RegexOptions.Singleline);
				MatchCollection mc = threadRe.Matches(postPage);
				int totalCount = mc.Count;
				lock (scarfInfo)
				{
					 scarfInfo.put("runtime.totalPostsFound", ((int)scarfInfo.get("runtime.totalPostsFound")) + totalCount);
				}
				if (pagePos != totalPages && (int)scarfInfo.get("expectedPostsPerPage") != mc.Count)
					writeTrace(app, scarfInfo, "<ForumScarf id='{0}' event='forumPostsFound-Issue1' numPosts='{1}' expectedPosts='{2}' url='{3}' pageSize='{4}' />",
						scarfInfo.get("id"), mc.Count, scarfInfo.get("expectedPostsPerPage"), url, threadPage.Length);
				
				for (int pos = 0; pos < totalCount; pos++)
				{
					Match m2 = mc[pos];
                    string postUserId = m2.Groups["userId"].Value;
                    if (ignoreUserIds.Contains("," + postUserId + ","))
                        continue;

                    string postId = m2.Groups["postId"].Value;
					string postDate = fixupDateStr(m2.Groups["postDate"].Value.Trim());
					string postUserName = fixupUserName(m2.Groups["userName"].Value);
					string postUserJoinDate = fixupDateStr(m2.Groups["joinDate"].Value.Trim());
                    string postUserAvitar = m2.Groups["avitar"].Value;
                    string postUserLocation = m2.Groups["location"].Value;
					string postTitle = m2.Groups["title"].Value.Trim();
					string postBody = filterBodyContent(app, context, scarfInfo, m2.Groups["body"].Value, (string)threadBizObj.get("uniqueTitle"), postUserName);
					IBizObj postBizObj = null;
					if (pos == 0 && pagePos == 1)
						postBizObj = threadBizObj;
					else
						postBizObj = posts.insertNew();

					Swaf.BGuid.Guid authorId = checkForUser(app, scarfInfo, postUserName, postUserId, postUserJoinDate, postUserLocation, postUserAvitar);
					postBizObj.put("counts.author.id", authorId);

					if (pos == 0 && pagePos == 1)
						postBizObj = setupForumBizObj(context, postBizObj, scarfInfo, postId, threadBizObj.get("parentId"), 3, null, null, postBody, postDate, true, authorId, true);
					else
						postBizObj = setupForumBizObj(context, postBizObj, scarfInfo, postId, threadBizObj.get("id"), 4, postTitle, null, postBody, postDate, false, authorId, true);
					totalPostsFound++;
				}
			}
			if (totalPostsFound != expectedPosts)
			{
				writeTrace(app, scarfInfo, "<ForumScarf id='{0}' event='forumPostsFound-Issue2' numPosts='{1}' expectedPosts='{2}' url='{3}' />",
					scarfInfo.get("id"), totalPostsFound, expectedPosts, url);
			}
		}

        public static char[] s_dirDiv = new char[]{'/', '\\'};
		public static Regex s_generalImgRegex = new Regex(@"<img.*?src=[""'](?'src'.*?)[""'].*?/>", RegexOptions.Singleline|RegexOptions.IgnoreCase);
		protected string filterBodyContent(IApplication app, IRuleContext context, IBizObj scarfInfo, string rawBody, string threadUniqueTitle, string userName)
		{
			string workBase = rawBody.Trim();

            MatchCollection mc = null;
            foreach (IBizObj replacement in (IBizObjList)scarfInfo.get("postReplacements"))
            {
                Regex rep = new Regex(replacement.get("regex").ToString());

                if ("linc".CompareTo(replacement.get("type")) == 0)
                {
                    mc = rep.Matches(workBase);
                    if (mc.Count != 0)
                    {
                        int lastPos = 0;
                        StringBuilder buff = new StringBuilder(workBase.Length);
                        foreach (Match m in mc)
                        {
                            string nextChunk = workBase.Substring(lastPos, m.Index - lastPos);
                            buff.Append(nextChunk);
                            lastPos = m.Index + m.Length;
                            buff.Append(getLincReplacement(m));
                        }
                        buff.Append(workBase.Substring(lastPos));
                        workBase = buff.ToString();
                    }
                }
                else
                    workBase = rep.Replace(workBase, replacement.get("replace").ToString());
            }

            string imgCache = "/"+ (string)scarfInfo.get("imageCache") + "/";
			string re = (string)scarfInfo.get("postSiteImageRegex");
			if (!Cmn.IsEmpty(re))
			{
				Regex imgRegex = new Regex((string)scarfInfo.get("postSiteImageRegex"), RegexOptions.Singleline);
				mc = imgRegex.Matches(workBase);
				if (mc.Count != 0)
				{
					StringBuilder buff = new StringBuilder(workBase.Length);
					int lastPos = 0;
					lock (scarfInfo)
					{
						scarfInfo.put("runtime.totalImagesFound", ((int)scarfInfo.get("runtime.totalImagesFound")) + mc.Count);
					}
					foreach (Match m in mc)
					{
						string nextChunk = workBase.Substring(lastPos, m.Index - lastPos);
						buff.Append(nextChunk);
						lastPos = m.Index + m.Length;
						string origImg = string.Format("{0}{1}.{2}", m.Groups["p1"].Value, m.Groups["p2"].Value, m.Groups["p3"].Value);
						string ourImg = string.Format("{0}.{1}", m.Groups["p2"].Value, m.Groups["p3"].Value);

						try
						{
							//Download image and add to desired media cache.
							string inCacheDir = userName.Replace(".","");
							foreach (IBizObj imgLoc in (IBizObjList)scarfInfo.get("imageLocations"))
							{
								Regex ir = new Regex((string)imgLoc.get("regex"));
								if (ir.Match(origImg).Success)
								{
									inCacheDir = (string)imgLoc.get("dir");
									break;
								}
							}
							ourImg = inCacheDir + "/" + ourImg.Substring(ourImg.LastIndexOf("/") + 1);

							downloadImageIntoCache(app, scarfInfo, origImg, ourImg);
                            setupImageForLincRef(app, scarfInfo, m, ourImg);
							string imageRes = "Thumbnail/";
							string newImageStructure = ((string)scarfInfo.get("newImageStructure")).Replace("$ourImageName$", ourImg).Replace("$imageRes$", imageRes);
							buff.Append(newImageStructure);
							lock (scarfInfo)
							{
								IMap rm = scarfInfo.get("runtime.threadsWithImages") as IMap;
								if (rm.containsField(threadUniqueTitle))
									rm[threadUniqueTitle] = (int)rm[threadUniqueTitle] + 1;
								else
									rm[threadUniqueTitle] = 1;
							}
						}
						catch (Exception e)
						{
							buff.Append(workBase.Substring(m.Index, m.Length));
						}
					}
					buff.Append(workBase.Substring(lastPos));
					workBase = buff.ToString();
				}
			}

			//Now look for general images
			mc = s_generalImgRegex.Matches(workBase);
			if (mc.Count != 0)
			{
				StringBuilder buff = new StringBuilder(workBase.Length);
				int lastPos = 0;
				lock (scarfInfo)
				{
					scarfInfo.put("runtime.totalImagesFound", ((int)scarfInfo.get("runtime.totalImagesFound")) + mc.Count);
				}
				string[] imgParts = ((string)scarfInfo.get("leaveImageOrigSizeIfContains")).Split(',');
				foreach (Match m in mc)
				{
					string nextChunk = workBase.Substring(lastPos, m.Index - lastPos);
					buff.Append(nextChunk);
					lastPos = m.Index + m.Length;
					string origImg = m.Groups["src"].Value;
                    string ourImg = null;

                    //This would happen from matching an image placed from the first image search loop above.
                    if (origImg.Contains(imgCache))
                    {
                        buff.Append(m.Groups[0].Value);
                        continue;
                    }

                    if (origImg.StartsWith("http"))
                    {
                        Uri imgUri = new Uri(origImg);
                        ourImg = string.Format("{0}/{1}", imgUri.Host, imgUri.LocalPath);
                    }
                    else
                    {
                        ourImg = origImg;
                        origImg = (string)scarfInfo.get("baseSiteUrl") + "/" + origImg;
                    }

                        try
                        {
                            //Download image and add to desired media cache.
                            string inCacheDir = userName;
                            foreach (IBizObj imgLoc in (IBizObjList)scarfInfo.get("imageLocations"))
                            {
                                Regex ir = new Regex((string)imgLoc.get("regex"));
                                if (ir.Match(origImg).Success)
                                {
                                    inCacheDir = (string)imgLoc.get("dir");
                                    break;
                                }
                            }
                            ourImg = inCacheDir + "/" + ourImg.Substring(ourImg.LastIndexOf("/") + 1);
                            downloadImageIntoCache(app, scarfInfo, origImg, ourImg);
                            string imageRes = "Thumbnail/";
                            foreach (string part in imgParts)
                                if (origImg.Contains(part))
                                {
                                    imageRes = "";
                                    break;
                                }

                            string newImageStructure = null;
                            if(Cmn.IsEmpty(imageRes))
                                newImageStructure = ((string)scarfInfo.get("newImageStructureNoLink")).Replace("$ourImageName$", ourImg).Replace("$imageRes$", imageRes);
                            else
                                newImageStructure = ((string)scarfInfo.get("newImageStructure")).Replace("$ourImageName$", ourImg).Replace("$imageRes$", imageRes);
                            buff.Append(newImageStructure);
                            lock (scarfInfo)
                            {
                                IMap rm = scarfInfo.get("runtime.threadsWithImages") as IMap;
                                if (rm.containsField(threadUniqueTitle))
                                    rm[threadUniqueTitle] = (int)rm[threadUniqueTitle] + 1;
                                else
                                    rm[threadUniqueTitle] = 1;
                            }
                        }
                        catch (Exception e)
                        {
                            buff.Append(workBase.Substring(m.Index, m.Length));
                        }
				}
				buff.Append(workBase.Substring(lastPos));
				workBase = buff.ToString();
			}

			//TODO: Look for smilies in workBase

			return workBase;
		}

        //this section is only used for LinC scrapping, but shows that there is a need for the generalized scarf mechanism to know how to call a method 
        //on an external app. Fix this later.
        static FlexiMap m = new FlexiMap();
        protected void setupImageForLincRef(IApplication app, IBizObj scarfInfo, Match matchInfo, string ourImg)
        {
            string title = matchInfo.Groups["title"].Value;
            if (!Cmn.IsEmpty(title))
            {
                int ind = title.LastIndexOf('-');
                if (ind != -1)
                {
                    string imgName = title.Substring(0, ind).Trim().Replace('.', '@');
                    m.put(imgName, string.Format("/img/{0}/{1}", scarfInfo.get("imageCache"), ourImg), "expire in 5 min");
                }
            }
        }

        static string s_lincImgRef = "<a href='{0}' target='_blank'>{1}</a>";
        protected string getLincReplacement(Match matchInfo)
        {
            string rep = "";
            string title = matchInfo.Groups["title"].Value;
            string key = title.Replace('.', '@');
            if (!Cmn.IsEmpty(title) && m.containsField(key))
            {
                string ourImg = m.get(key).ToString();
                rep = string.Format(s_lincImgRef, ourImg, title);
            }
            return rep;
        }
        //--------------------------------------------------------------------------------------------------------------

		public void filterExistingForumPosts(IApplication app, IRuleContext context, IBizObj scarfInfo)
		{
			Swaf.BGuid.Guid forumSetArticleId = context.RuleGroup.execute(context, "getArticleIdFromUniqueTitle", scarfInfo.get("id") + "0_", "Forum").CallResults as Swaf.BGuid.Guid;

			FlexiMap m = new FlexiMap();
			m.put("objectTreeName", "Forum");
			m.put("companyId", ActiveCompanyId);
			m.put("parentId", forumSetArticleId);
			m.put("entireTree", true);
			m.put("listDetailLevel", DetailLevel.fullDetails);
			//m.put("isPaging", true);
			//m.put("UseDarNamed", "DA.Articles.GetArticlesForumAll");

			IResults rc = app.ruleMgr.execute("getArticles", m, null);
			IBizObjList posts = rc.CallResults as IBizObjList;
			int index = 0;
			foreach (IBizObj post in posts)
			{
				string origBody = (string)post.get("baseBody");
				string newBody = filterBodyContent(app, context, scarfInfo, origBody, (string)post.get("uniqueTitle"), post.get("counts.author.userName").ToString());
				if (newBody.CompareTo(origBody) != 0)
				{
					//object articleId,bool withTranslation,DetailLevel detailLevel, bool incrementCount, bool stripLeading)
					IBizObj fullDetail = context.RuleGroup.execute(context, "getArticleDetail",
						post.get("id"), false, DetailLevel.fullDetails, false, false, null).CallResults as IBizObj;
					fullDetail.put("baseBody", newBody);
					setupContext(context, true);
					app.ruleMgr.execute(context, "updateArticle", fullDetail);
				}
				index++;
			}
		}


		protected Swaf.BGuid.Guid checkForUser(IApplication app, IBizObj scarfInfo, string userName, string extUserId, string dateJoined, string location, string avitar)
		{
			Swaf.BGuid.Guid userId = null;
            string realUserName = scarfInfo.get("userIdPrefix") + "." + userName;
            string realExtId = scarfInfo.get("userIdPrefix") + "." + extUserId;
			IMap m = scarfInfo.get("runtime.uniqueUsers") as IMap;

            IBizObj userInfo = null;
            IBizObj contactInfo = app.ruleMgr.execute("getContactDetailFromExtId", realExtId).CallResults as IBizObj;
            if (contactInfo == null)
            {
                userInfo = app.ruleMgr.execute("getUserFromUserName", realUserName).CallResults as IBizObj;
                if (userInfo == null)
                {
                    IBizObj user = app.bizObjMgr.create("SignUpInfo");
                    user.put("productKey", "UM-BasicMember-NB");
                    user.put("rootUserInfo.id", userId = new Swaf.BGuid.Guid());
                    user.put("rootUserInfo.userName", realUserName);
                    user.put("rootUserInfo.password", "P@ssword");
                    user.put("rootUserInfo.password2", "P@ssword");
                    user.put("contactInfo.id", user.get("rootUserInfo.id"));
                    DateTime joined = DateTime.UtcNow;
                    if (!Cmn.IsEmpty(dateJoined))
                        DateTime.TryParse(dateJoined.Trim(), out joined);
                    user.put("rootUserInfo.dateCreated", joined);
                    user.put("rootUserInfo.dateUpdated", DateTime.UtcNow);
                    IResults rc = app.ruleMgr.execute("processNewUserSignUp", user, null, false);
                }
                else
                    userId = userInfo.get("id") as Swaf.BGuid.Guid;

            }
            else
                userId = contactInfo.get("id") as Swaf.BGuid.Guid;

            if (contactInfo == null)
            {
                //Try to get contact info based on userId as sometimes the external id isnt present in the db
                if (!userInfo.isNew)
                    contactInfo = app.ruleMgr.execute("getContactDetail", userId).CallResults as IBizObj;
                if (contactInfo == null)
                {
                    contactInfo = app.bizObjMgr.create("Contact");
                    contactInfo.put("id", userId);
                    contactInfo.put("type.id", App.appData["contactOptions.types.(name:Customer).id"]);
                    contactInfo.put("companyId", ActiveCompanyId);
                }
            }

            contactInfo.put("extId", realExtId);
            if (!((string)scarfInfo.get("leaveExtContactOptionsAlone")).Contains("," + extUserId + ","))
            {
                contactInfo.put("isUser", true);
                contactInfo.put("location", location);
                string ourImg = "";
                if (!Cmn.IsEmpty(avitar) && !avitar.StartsWith("http://www.gravatar.com/"))
                {
                    if (!avitar.StartsWith("http", StringComparison.CurrentCultureIgnoreCase))
                        avitar = scarfInfo.get("baseSiteUrl") + "/" + avitar;
                    ourImg = userName + "/" + avitar.Substring(avitar.LastIndexOf('/') + 1);
                    downloadImageIntoCache(app, scarfInfo, avitar, ourImg);
                }
                contactInfo.put("avitarImage", ourImg);
            }
            IResults rc2 = app.ruleMgr.execute("updateContact", contactInfo, false);

			lock (scarfInfo)
			{
				string key = realUserName.Replace('.', '_');
				if (m.containsField(key))
					m.put(key, (int)m.get(key) + 1);
				else
					m.put(key, 1);
				if(userInfo == null)
					scarfInfo.put("runtime.totalNewUsersFound", ((int)scarfInfo.get("runtime.totalNewUsersFound")) + 1);
			}

            return userId;
		}
	}
}
